{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "import gzip\n",
    "import json\n",
    "import numpy as np\n",
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_DIR = os.path.expanduser(\"~/blob/raw_datasets/steam\")\n",
    "\n",
    "review_file = os.path.join(DATA_DIR, \"steam_reviews.json.gz\")\n",
    "meta_file = os.path.join(DATA_DIR, \"steam_games.json.gz\")\n",
    "\n",
    "all_meta_file = os.path.join(DATA_DIR, \"steam_games.csv\")\n",
    "\n",
    "processed_raw_folder = os.path.join(DATA_DIR, \"processed\")\n",
    "raw_review_path = os.path.join(processed_raw_folder, 'reviews.csv')\n",
    "raw_meta_path = os.path.join(processed_raw_folder, 'meta.csv')\n",
    "\n",
    "chatbot_data_folder = os.path.join(DATA_DIR, 'chatbot')\n",
    "\n",
    "if not os.path.exists(processed_raw_folder):\n",
    "    os.makedirs(processed_raw_folder)\n",
    "\n",
    "if not os.path.exists(chatbot_data_folder):\n",
    "    os.makedirs(chatbot_data_folder)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# with open(os.path.join(DATA_DIR, \"steam_games.json\"), 'w') as f:\n",
    "#     games = json.load(f)\n",
    "\n",
    "def parse(path):\n",
    "    g = gzip.open(path, 'r')\n",
    "    for l in g:\n",
    "        yield eval(l)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(raw_review_path):\n",
    "    raw_review_df = pd.DataFrame(list(parse(review_file)))\n",
    "    raw_review_df.to_csv(raw_review_path, index=None, sep='|')\n",
    "else:\n",
    "    raw_review_df = pd.read_csv(raw_review_path, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(raw_meta_path):\n",
    "    raw_meta_df = pd.DataFrame(list(parse(meta_file)))\n",
    "    raw_meta_df.to_csv(raw_meta_path, index=None, sep='|')\n",
    "else:\n",
    "    raw_meta_df = pd.read_csv(raw_meta_path, sep='|')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_meta_df = pd.read_csv(all_meta_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['username', 'hours', 'products', 'product_id', 'page_order', 'date',\n",
      "       'text', 'early_access', 'page', 'found_funny', 'compensation',\n",
      "       'user_id'],\n",
      "      dtype='object')\n",
      "               username  hours  products  product_id  page_order        date  \\\n",
      "3431749  Indifferential   15.3      93.0      282140           5  2016-11-24   \n",
      "3789391          Kadesh  804.5      15.0      433850           7  2017-01-08   \n",
      "5883488         UniDoge   28.3      35.0      202530           2  2015-05-05   \n",
      "\n",
      "                                                      text  early_access  \\\n",
      "3431749  SOMA is a story-driven sci-fi horror game.Ther...         False   \n",
      "3789391                              Vicia mais que crack.          True   \n",
      "5883488  This game is iffy for me.\\nI've got pros, most...         False   \n",
      "\n",
      "         page  found_funny compensation  user_id  \n",
      "3431749   310          NaN          NaN      NaN  \n",
      "3789391  2801          NaN          NaN      NaN  \n",
      "5883488    45          NaN          NaN      NaN  \n"
     ]
    }
   ],
   "source": [
    "print(raw_review_df.columns)\n",
    "print(raw_review_df.sample(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['publisher', 'genres', 'app_name', 'title', 'url', 'release_date',\n",
      "       'tags', 'discount_price', 'reviews_url', 'specs', 'price',\n",
      "       'early_access', 'id', 'developer', 'sentiment', 'metascore'],\n",
      "      dtype='object')\n",
      "         publisher      genres                     app_name  \\\n",
      "1357   Deep Silver  ['Action']  Sacred Citadel: Jungle Hunt   \n",
      "3851     DYA Games   ['Indie']   Super Star Path Soundtrack   \n",
      "31377     2K Games  ['Action']   Mafia II DLC: Greaser Pack   \n",
      "\n",
      "                             title  \\\n",
      "1357   Sacred Citadel: Jungle Hunt   \n",
      "3851    Super Star Path Soundtrack   \n",
      "31377   Mafia II DLC: Greaser Pack   \n",
      "\n",
      "                                                     url release_date  \\\n",
      "1357   http://store.steampowered.com/app/222380/Sacre...   2013-04-22   \n",
      "3851   http://store.steampowered.com/app/378910/Super...   2015-06-22   \n",
      "31377  http://store.steampowered.com/app/67330/Mafia_...   2010-09-28   \n",
      "\n",
      "                                                    tags  discount_price  \\\n",
      "1357                                          ['Action']             NaN   \n",
      "3851   ['Indie', 'Music', 'Arcade', 'Retro', \"Shoot '...            0.89   \n",
      "31377                                         ['Action']             NaN   \n",
      "\n",
      "                                             reviews_url  \\\n",
      "1357   http://steamcommunity.com/app/222380/reviews/?...   \n",
      "3851   http://steamcommunity.com/app/378910/reviews/?...   \n",
      "31377  http://steamcommunity.com/app/67330/reviews/?b...   \n",
      "\n",
      "                                                   specs price  early_access  \\\n",
      "1357   ['Single-player', 'Multi-player', 'Downloadabl...  1.99         False   \n",
      "3851           ['Single-player', 'Downloadable Content']  0.99         False   \n",
      "31377  ['Single-player', 'Downloadable Content', 'Ste...  2.99         False   \n",
      "\n",
      "             id  developer       sentiment  metascore  \n",
      "1357   222380.0   Southend  9 user reviews        NaN  \n",
      "3851   378910.0  DYA Games             NaN        NaN  \n",
      "31377   67330.0   2K Czech  1 user reviews        NaN  \n"
     ]
    }
   ],
   "source": [
    "print(raw_meta_df.columns)\n",
    "print(raw_meta_df.sample(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['url', 'types', 'name', 'desc_snippet', 'recent_reviews', 'all_reviews',\n",
      "       'release_date', 'developer', 'publisher', 'popular_tags',\n",
      "       'game_details', 'languages', 'achievements', 'genre',\n",
      "       'game_description', 'mature_content', 'minimum_requirements',\n",
      "       'recommended_requirements', 'original_price', 'discount_price'],\n",
      "      dtype='object')\n",
      "                                                     url types  \\\n",
      "35969  https://store.steampowered.com/app/596160/Tric...   app   \n",
      "40199  https://store.steampowered.com/app/773806/The_...   app   \n",
      "38265  https://store.steampowered.com/app/1033080/Let...   app   \n",
      "\n",
      "                                                    name  \\\n",
      "35969                            TrickStyle - Soundtrack   \n",
      "40199  The Legend of Heroes: Trails of Cold Steel II ...   \n",
      "38265                      Letters - a written adventure   \n",
      "\n",
      "                                            desc_snippet recent_reviews  \\\n",
      "35969                                                NaN            NaN   \n",
      "40199                                                NaN            NaN   \n",
      "38265  Letters - a written adventure is a multilinear...            NaN   \n",
      "\n",
      "      all_reviews  release_date                developer  \\\n",
      "35969         NaN   Sep 9, 1999  Throwback Entertainment   \n",
      "40199         NaN  Feb 14, 2018             Nihon Falcom   \n",
      "38265         NaN          2020           5am Games GmbH   \n",
      "\n",
      "                                               publisher  \\\n",
      "35969  Throwback Entertainment ,Throwback Entertainment    \n",
      "40199  XSEED Games,Marvelous USA, Inc.,XSEED Games,Ma...   \n",
      "38265                      5am Games GmbH,5am Games GmbH   \n",
      "\n",
      "                                            popular_tags  \\\n",
      "35969                               Action,Racing,Sports   \n",
      "40199                                                RPG   \n",
      "38265  Adventure,Indie,Cute,Female Protagonist,Word G...   \n",
      "\n",
      "                                            game_details         languages  \\\n",
      "35969  Single-player,Downloadable Content,Profile Fea...           English   \n",
      "40199  Single-player,Downloadable Content,Steam Achie...  English,Japanese   \n",
      "38265  Single-player,Steam Achievements,Full controll...           English   \n",
      "\n",
      "       achievements                 genre  \\\n",
      "35969           NaN  Action,Racing,Sports   \n",
      "40199           NaN                   RPG   \n",
      "38265           NaN       Adventure,Indie   \n",
      "\n",
      "                                        game_description mature_content  \\\n",
      "35969   About This Content Grab your hoverboard and t...            NaN   \n",
      "40199   About This Content Civil war has broken out a...            NaN   \n",
      "38265   About This Game  Letters is a single-player p...            NaN   \n",
      "\n",
      "                                    minimum_requirements  \\\n",
      "35969  Minimum:,OS:,Any,Processor:,2 Ghz,Memory:,1 MB...   \n",
      "40199                                                NaN   \n",
      "38265                                                NaN   \n",
      "\n",
      "                                recommended_requirements original_price  \\\n",
      "35969  Recommended:,OS:,Windows 7,Processor:,2+ Ghz,M...           Free   \n",
      "40199                                                NaN          $2.99   \n",
      "38265                                                NaN            NaN   \n",
      "\n",
      "      discount_price  \n",
      "35969            NaN  \n",
      "40199            NaN  \n",
      "38265            NaN  \n"
     ]
    }
   ],
   "source": [
    "print(all_meta_df.columns)\n",
    "print(all_meta_df.sample(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                                     DOOM\n",
       "1                            PLAYERUNKNOWN'S BATTLEGROUNDS\n",
       "2                                               BATTLETECH\n",
       "3                                                     DayZ\n",
       "4                                               EVE Online\n",
       "                               ...                        \n",
       "40828    Rocksmith® 2014 Edition – Remastered – Sabaton...\n",
       "40829    Rocksmith® 2014 Edition – Remastered – Stone T...\n",
       "40830    Fantasy Grounds - Quests of Doom 4: A Midnight...\n",
       "40831                         Mega Man X5 Sound Collection\n",
       "40832                                     Stories In Stone\n",
       "Name: name, Length: 40833, dtype: object"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_meta_df['name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coverage:  0.6236797008879887\n"
     ]
    }
   ],
   "source": [
    "raw_game_names = raw_meta_df['app_name'].unique()\n",
    "all_game_names = all_meta_df['name'].unique()\n",
    "\n",
    "print(\"Coverage: \", len([g for g in raw_game_names if g in all_game_names]) / len(raw_game_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Text Clean Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "This is a text,. including emojiand special symbols!\n"
     ]
    }
   ],
   "source": [
    "import re  \n",
    "from bs4 import BeautifulSoup  \n",
    "  \n",
    "def remove_html_tags(text):  \n",
    "    \"\"\"Remove HTML tags from text\"\"\"  \n",
    "    soup = BeautifulSoup(text, \"html.parser\")  \n",
    "    return soup.get_text()  \n",
    "  \n",
    "def remove_emojis(text):  \n",
    "    emoji_pattern = re.compile(  \n",
    "        \"[\"  \n",
    "        u\"\\U0001F600-\\U0001F64F\"  # emoticons  \n",
    "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs  \n",
    "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols  \n",
    "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)  \n",
    "        u\"\\U00002702-\\U000027B0\"  \n",
    "        u\"\\U000024C2-\\U0001F251\"  \n",
    "        \"]+\",  \n",
    "        flags=re.UNICODE,  \n",
    "    )  \n",
    "    return emoji_pattern.sub(r\"\", text)  \n",
    "  \n",
    "def remove_special_characters(text, remove_digits=False):  \n",
    "    pattern = r\"[^a-zA-Z0-9\\s\\u4e00-\\u9fa5\\,\\.\\!]\" if not remove_digits else r\"[^a-zA-Z\\s\\u4e00-\\u9fa5\\,\\.\\!]\"  \n",
    "    return re.sub(pattern, \"\", text)  \n",
    "  \n",
    "def clean_text(text, remove_digits=False):  \n",
    "    text = remove_html_tags(text)  \n",
    "    text = remove_emojis(text)  \n",
    "    text = remove_special_characters(text, remove_digits=remove_digits)  \n",
    "    return text  \n",
    "  \n",
    "# example\n",
    "text = \"<html><head></head><body><p>This is a text,.: including emoji👍and special symbols!@#</p></body></html>\"  \n",
    "cleaned_text = clean_text(text)  \n",
    "print(cleaned_text)  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Clean Text in Meta DF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"['Strategy', 'Action', 'Indie', 'Casual', 'Simulation']\""
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_meta_df['tags'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def process_price(x: str):\n",
    "    if isinstance(x, str):\n",
    "        # try:\n",
    "        price = re.findall(r\"\\d+\\.?\\d*\", x)\n",
    "        if price:\n",
    "            return eval(price[0])\n",
    "        else:\n",
    "            return 0\n",
    "    elif isinstance(x, float):\n",
    "        return x\n",
    "    else:\n",
    "        return 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_meta_df['price'] = raw_meta_df['price'].apply(process_price)\n",
    "avg_price = raw_meta_df['price'][~raw_meta_df['price'].isna()].mean()\n",
    "raw_meta_df['price'] = raw_meta_df['price'].fillna(avg_price)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title NAN:  2050\n",
      "App name NAN:  2\n",
      "0.9189357398475183\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>app_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Log Challenge</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Icarus Six Sixty Six</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>After Life VR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Kitty Hawk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Mortars VR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32073</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Tank of War-VR</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32076</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Flappy Arms</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32077</th>\n",
       "      <td>NaN</td>\n",
       "      <td>SpaceWalker</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32085</th>\n",
       "      <td>NaN</td>\n",
       "      <td>LIV Client</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32134</th>\n",
       "      <td>NaN</td>\n",
       "      <td>Maze Run VR</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2605 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      title              app_name\n",
       "4       NaN         Log Challenge\n",
       "11      NaN  Icarus Six Sixty Six\n",
       "19      NaN         After Life VR\n",
       "20      NaN            Kitty Hawk\n",
       "22      NaN            Mortars VR\n",
       "...     ...                   ...\n",
       "32073   NaN        Tank of War-VR\n",
       "32076   NaN           Flappy Arms\n",
       "32077   NaN           SpaceWalker\n",
       "32085   NaN            LIV Client\n",
       "32134   NaN           Maze Run VR\n",
       "\n",
       "[2605 rows x 2 columns]"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(\"Title NAN: \", (raw_meta_df['title'].isna().sum()))\n",
    "print(\"App name NAN: \", raw_meta_df['app_name'].isna().sum())\n",
    "print((raw_meta_df['app_name'] == raw_meta_df['title']).sum() / len(raw_meta_df))\n",
    "raw_meta_df[raw_meta_df['app_name'] != raw_meta_df['title']][['title', 'app_name']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "col2types = {\n",
    "    \"publisher\": str,\n",
    "    \"app_name\": str,\n",
    "    \"price\": float,\n",
    "    \"id\": int,\n",
    "    \"developer\": str,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_meta_df_0 = raw_meta_df[~raw_meta_df['id'].isna()]\n",
    "raw_meta_df_0 = raw_meta_df_0[~raw_meta_df_0['app_name'].isna()]\n",
    "raw_meta_df_0.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_meta_df_1 = raw_meta_df_0.astype(col2types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_215302/90167381.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")\n",
      "/tmp/ipykernel_215302/90167381.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")\n"
     ]
    }
   ],
   "source": [
    "raw_meta_df_1['app_name'] = raw_meta_df_1['app_name'].apply(lambda x: clean_text(x) if x is not None else \"\")\n",
    "raw_meta_df_1['developer'] = raw_meta_df_1['developer'].apply(lambda x: clean_text(x) if x is not None else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_215302/90167381.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_215302/90167381.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")\n",
      "/tmp/ipykernel_215302/90167381.py:6: MarkupResemblesLocatorWarning: The input looks more like a filename than markup. You may want to open this file and pass the filehandle into Beautiful Soup.\n",
      "  soup = BeautifulSoup(text, \"html.parser\")\n"
     ]
    }
   ],
   "source": [
    "all_meta_df_0 = all_meta_df[~all_meta_df['name'].isna()].reset_index(drop=True)\n",
    "all_meta_df_0 = all_meta_df_0.astype({'name': 'str', 'game_details': 'str', 'game_description': 'str'})\n",
    "all_meta_df_0['name'] = all_meta_df_0['name'].apply(lambda x: clean_text(x) if x is not None else \"\")\n",
    "all_meta_df_0['game_details'] = all_meta_df_0['game_details'].apply(lambda x: clean_text(x) if x is not None else \"\")\n",
    "all_meta_df_0['game_description'] = all_meta_df_0['game_description'].apply(lambda x: clean_text(x) if x is not None else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_meta_df_0[\"_lower_name\"] = all_meta_df_0['name'].apply(lambda x: x.lower())\n",
    "raw_meta_df_1[\"_lower_name\"] = raw_meta_df_1['app_name'].apply(lambda x: x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_meta_df_0[\"_lower_name\"] = all_meta_df_0[\"_lower_name\"].astype(str)\n",
    "raw_meta_df_1[\"_lower_name\"] = raw_meta_df_1[\"_lower_name\"].astype(str)\n",
    "raw_meta_df_1 = raw_meta_df_1.drop_duplicates(keep='first').reset_index(drop=True)\n",
    "raw_meta_df_1 = raw_meta_df_1.set_index('_lower_name')\n",
    "all_meta_df_0 = all_meta_df_0.set_index('_lower_name')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_df = raw_meta_df_1.join(all_meta_df_0, on='_lower_name', rsuffix='_all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11991 11991\n",
      "3628 162\n"
     ]
    }
   ],
   "source": [
    "print(cat_df['game_description'].isna().sum(), cat_df['game_details'].isna().sum())\n",
    "print(cat_df['genres'].isna().sum(), cat_df['tags'].isna().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['publisher', 'genres', 'app_name', 'title', 'url', 'release_date',\n",
       "       'tags', 'discount_price', 'reviews_url', 'specs', 'price',\n",
       "       'early_access', 'id', 'developer', 'sentiment', 'metascore', 'url_all',\n",
       "       'types', 'name', 'desc_snippet', 'recent_reviews', 'all_reviews',\n",
       "       'release_date_all', 'developer_all', 'publisher_all', 'popular_tags',\n",
       "       'game_details', 'languages', 'achievements', 'genre',\n",
       "       'game_description', 'mature_content', 'minimum_requirements',\n",
       "       'recommended_requirements', 'original_price', 'discount_price_all'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_df['release_date_all'] = pd.to_datetime(cat_df['release_date_all'], format=\"%b %d, %Y\", errors=\"coerce\")\n",
    "cat_df['release_date'] = pd.to_datetime(cat_df['release_date'], format=\"%Y-%m-%d\", errors=\"coerce\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_215302/3082042013.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cat_df['release_date'][cat_df['release_date'].isna()] = cat_df['release_date_all'][cat_df['release_date'].isna()]\n",
      "/tmp/ipykernel_215302/3082042013.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  cat_df['release_date'][cat_df['release_date'].isna()] = cat_df['release_date_all'][cat_df['release_date'].isna()]\n"
     ]
    }
   ],
   "source": [
    "cat_df['release_date'][cat_df['release_date'].isna()] = cat_df['release_date_all'][cat_df['release_date'].isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "used_cols = [\"id\", \"app_name\", \"release_date\", \"tags\", \"price\", \"game_description\"]\n",
    "final_meta_df = cat_df[used_cols]\n",
    "final_meta_df = final_meta_df.reset_index(drop=True)\n",
    "final_meta_df['game_description'] = final_meta_df['game_description'].fillna(\"No description\")\n",
    "final_meta_df.rename(columns={'app_name': 'title', 'game_description': \"description\"}, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(           id                           title release_date  \\\n",
       " 5917   427250                     8Bit Armies   2016-04-22   \n",
       " 33235   23400   Imperium Romanum Gold Edition   2008-10-30   \n",
       " 28975  345860  Super Galaxy Squadron EX Turbo   2017-03-06   \n",
       " \n",
       "                                                     tags  price  \\\n",
       " 5917   ['Strategy', 'RTS', 'Pixel Graphics', 'Simulat...  14.99   \n",
       " 33235  ['Strategy', 'Simulation', 'City Builder', 'Ro...   9.99   \n",
       " 28975  ['Indie', \"Shoot 'Em Up\", 'Retro', 'Action', '...  11.99   \n",
       " \n",
       "                                              description  \n",
       " 5917    About This Game  A Blast From the Past! 8Bit ...  \n",
       " 33235   About This Game The city builder strategy gam...  \n",
       " 28975                                     No description  ,\n",
       " (33595, 6))"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_meta_df.sample(3), final_meta_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(raw_review_df['product_id'].isin(final_meta_df['id'])).sum() / raw_review_df.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "review_df_0 = raw_review_df[['username', 'product_id', 'date']]\n",
    "review_df_0 = review_df_0[~review_df_0['username'].isna()].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_valid_ids(df, col_name, k):\n",
    "        frequency = df.groupby([col_name])[[col_name]].count()\n",
    "        valid_id = frequency[frequency[col_name]>=k].index\n",
    "        return valid_id\n",
    "\n",
    "def keepFirstFilter(df: pd.DataFrame, user_col: str='user_id', item_col: str='item_id', time_col: str='timestamp') -> pd.DataFrame:\n",
    "    print(f\"*** Only keep the first interaction for duplicated review ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    df = df.sort_values(by=[user_col, time_col], ).reset_index(drop=True)\n",
    "    df = df.drop_duplicates(subset=[user_col, item_col], keep='first').reset_index(drop=True)\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df\n",
    "\n",
    "\n",
    "def kCoreFilter(df: pd.DataFrame, user_k: int=10, item_k: int=10, user_col: str='user_id', item_col: str='item_id', max_iter: int=20) -> pd.DataFrame:\n",
    "    print(f\"*** Users whose interaction < {user_k} and items whose interaction < {item_k}  would be filtered out ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    \n",
    "    num_users_prev, num_items_prev = len(df[user_col].unique()), len(df[item_col].unique()) \n",
    "    delta = True\n",
    "    iter = 0\n",
    "    \n",
    "    while delta and iter < max_iter: \n",
    "        valid_users = get_valid_ids(df, user_col, user_k)  \n",
    "        df = df[df[user_col].isin(valid_users)]\n",
    "\n",
    "        valid_items = get_valid_ids(df, item_col, item_k)  \n",
    "        df = df[df[item_col].isin(valid_items)]\n",
    "\n",
    "        num_users = len(valid_users)\n",
    "        num_items = len(valid_items)  \n",
    "\n",
    "        delta = (num_users != num_users_prev) or (num_items != num_items_prev)\n",
    "        print('Iter: {0}, users: {1} / {2}, items: {3} / {4}'.format(iter, num_users, num_users_prev, num_items, num_items_prev))\n",
    "\n",
    "        num_users_prev = num_users\n",
    "        num_items_prev = num_items\n",
    "        iter+=1\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df \n",
    "\n",
    "\n",
    "def lowRatingFilter(df: pd.DataFrame, rating_thres: float=3.0, rating_col: str='rating') -> pd.DataFrame:\n",
    "    print(f\"*** Rating < {rating_thres} would be filtered out ***\")\n",
    "    print(f\"Begin: {df.shape}\")\n",
    "    df = df[df[rating_col] >= rating_thres].reset_index(drop=True)\n",
    "    print(f\"After: {df.shape}\")\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Only keep the first interaction for duplicated review ***\n",
      "Begin: (7792828, 3)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After: (6889528, 3)\n",
      "*** Users whose interaction < 5 and items whose interaction < 5  would be filtered out ***\n",
      "Begin: (6889528, 3)\n",
      "Iter: 0, users: 281639 / 2567530, items: 11978 / 15474\n",
      "Iter: 1, users: 281208 / 281639, items: 11961 / 11978\n",
      "Iter: 2, users: 281204 / 281208, items: 11961 / 11961\n",
      "Iter: 3, users: 281204 / 281204, items: 11961 / 11961\n",
      "After: (3484497, 3)\n"
     ]
    }
   ],
   "source": [
    "review_df_1 = keepFirstFilter(review_df_0, user_col='username', item_col='product_id', time_col='date')\n",
    "# review_df1 = lowRatingFilter(review_df_1, 3.0)\n",
    "review_df_1 = kCoreFilter(review_df_1, 5, 5, user_col='username', item_col='product_id')\n",
    "data_df = review_df_1.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = data_df.rename(columns={'username': 'user_id', 'product_id': 'item_id'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_id(df: pd.DataFrame, user_colname: str='user_id', item_colname: str='item_id', group_colname: str=None, return_map: bool=False, price_df: pd.DataFrame=None, group_df: pd.DataFrame=None) -> pd.DataFrame:\n",
    "    '''map user and item ids'''\n",
    "    users = df[user_colname].unique()\n",
    "    items = df[item_colname].unique()\n",
    "    print(\"**Map users and items**\")\n",
    "    n_users, n_items = len(users), len(items)\n",
    "    user_map = {u: k+1 for k, u in enumerate(users)}\n",
    "    item_map = {i: k+1 for k, i in enumerate(items)}\n",
    "    df[user_colname] = df[user_colname].apply(lambda x: user_map[x])\n",
    "    df[item_colname] = df[item_colname].apply(lambda x: item_map[x])\n",
    "\n",
    "    if (group_colname is not None) and (group_colname in df):\n",
    "        groups = df[group_colname].unique()\n",
    "        n_groups = len(groups)\n",
    "        group_map = {g: k+1 for k,g in enumerate(groups)}\n",
    "        df[group_colname] = df[group_colname].apply(lambda x: group_map[x])\n",
    "    else:\n",
    "        group_map = {}\n",
    "\n",
    "    if price_df is not None:\n",
    "        if user_colname in price_df:\n",
    "            price_df = price_df[price_df[user_colname].isin(users)].reset_index(drop=True)\n",
    "            price_df[user_colname] = price_df.apply(lambda x: user_map[x])\n",
    "        if item_colname in price_df:\n",
    "            price_df = price_df[price_df[item_colname].isin(items)].reset_index(drop=True)\n",
    "            price_df[item_colname] = price_df[item_colname].apply(lambda x: item_map[x])\n",
    "\n",
    "    if group_df is not None:\n",
    "        if user_colname in group_df:\n",
    "            group_df = group_df[group_df[user_colname].isin(users)].reset_index(drop=True)\n",
    "            group_df[user_colname] = group_df.apply(lambda x: user_map[x])\n",
    "        if item_colname in group_df:\n",
    "            group_df = group_df[group_df[item_colname].isin(items)].reset_index(drop=True)\n",
    "            group_df[item_colname] = group_df[item_colname].apply(lambda x: item_map[x])\n",
    "        if group_colname is not None and group_colname in group_df:\n",
    "            groups = group_df[group_colname].unique()\n",
    "            n_groups = len(groups)\n",
    "            group_map = {g: k+1 for k,g in enumerate(groups)}\n",
    "            group_df[group_colname] = group_df[group_colname].apply(lambda x: group_map[x])\n",
    "\n",
    "    if return_map:\n",
    "        return (df, price_df, group_df), (user_map, item_map, group_map)\n",
    "    else:\n",
    "        return df, price_df, group_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "**Map users and items**\n"
     ]
    }
   ],
   "source": [
    "(df, price_df, group_df),(user_map, item_map, group_map) = map_id(data_df, return_map=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_map_json = {str(k): v for k,v in item_map.items()}\n",
    "all_map = {'item': item_map_json, 'user': user_map}\n",
    "\n",
    "with open(os.path.join(processed_raw_folder, \"map.json\"), 'w') as f:\n",
    "    json.dump(all_map, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "# split\n",
    "def split_train_test_set_leave_one_out_seq(data: pd.DataFrame, col_name: str, time_colname:str, col_names_2_return: list, seed: int=42):\n",
    "    '''Leave the last one item for test set'''\n",
    "    if time_colname in data:\n",
    "        df_sorted = data.sort_values(by=[col_name, time_colname]).reset_index(drop=True)\n",
    "    else:\n",
    "        df_sorted = data.sort_values(by=col_name).reset_index(drop=True)\n",
    "\n",
    "    df_test = df_sorted.groupby(by=col_name, as_index=False).nth(-1)\n",
    "    df_train = df_sorted.iloc[df_sorted.index.difference(df_test.index)]\n",
    "    return df_train.reset_index(drop=True)[col_names_2_return], df_test.reset_index(drop=True)[col_names_2_return]\n",
    "\n",
    "def split_train_test_set_leave_one_out(data: pd.DataFrame, col_name: str, col_names_2_return: list, seed: int = 0):\n",
    "    if col_names_2_return is None:\n",
    "        col_names_2_return = data.columns #.to_list()\n",
    "    df_groupby = data.groupby(by=col_name, as_index=False) \n",
    "    df_test = df_groupby.sample(n=1, random_state=seed)[col_names_2_return]\n",
    "    df_train = data.iloc[data.index.difference(df_test.index)][col_names_2_return] \n",
    "    return df_train.reset_index(drop=True), df_test.reset_index(drop=True) \n",
    "\n",
    "def split_train_test_set_by_ratio(data: pd.DataFrame, ratio: list, col_name: str, col_names_2_return: list, seed: int = 0):\n",
    "    if col_names_2_return is None:\n",
    "        col_names_2_return = data.columns #.to_list()\n",
    "    assert len(ratio) == 2, 'ratio is for train/test.'\n",
    "    frac = ratio[1] / sum(ratio)\n",
    "    df_groupby = data.groupby(by=col_name, as_index=False) \n",
    "    df_test = df_groupby.sample(frac=frac, random_state=seed)[col_names_2_return]\n",
    "    df_train = data.iloc[data.index.difference(df_test.index)] \n",
    "    return df_train.reset_index(drop=True), df_test.reset_index(drop=True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_0, df_test = split_train_test_set_leave_one_out_seq(df, 'user_id', 'date', ['user_id', 'item_id'])\n",
    "df_train, df_valid = split_train_test_set_leave_one_out_seq(df_train_0, 'user_id', 'date', ['user_id', 'item_id',])\n",
    "\n",
    "df_train.to_csv(os.path.join(chatbot_data_folder, \"train.tsv\"), index=None)\n",
    "df_valid.to_csv(os.path.join(chatbot_data_folder, \"valid.tsv\"), index=None)\n",
    "df_test.to_csv(os.path.join(chatbot_data_folder, \"test.tsv\"), index=None)\n",
    "df_train_0.to_csv(os.path.join(chatbot_data_folder, \"user_history.tsv\"), index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df = final_meta_df[final_meta_df['id'].isin(item_map.keys())]\n",
    "saved_meta_df = saved_meta_df.drop_duplicates(subset=['id'], keep='first')\n",
    "saved_meta_df.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df['id'] = saved_meta_df['id'].apply(lambda x: item_map[x])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_meta_df.shape[0] == len(item_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df['tags'] = saved_meta_df['tags'].apply(lambda x: eval(x) if isinstance(x, str) else [])\n",
    "saved_meta_df.to_feather(os.path.join(chatbot_data_folder, 'games.ftr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['title', 'release_date', 'tags', 'price', 'description', 'visited_num'], dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "saved_meta_df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test Data for RecBot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_meta_df = pd.read_feather(os.path.join(chatbot_data_folder, 'games.ftr'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv(os.path.join(chatbot_data_folder, \"train.tsv\"))\n",
    "df_valid = pd.read_csv(os.path.join(chatbot_data_folder, \"valid.tsv\"))\n",
    "df_test = pd.read_csv(os.path.join(chatbot_data_folder, \"test.tsv\"))\n",
    "df_train_0 = pd.read_csv(os.path.join(chatbot_data_folder, \"user_history.tsv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_count = pd.value_counts(df_train_0['item_id'])\n",
    "saved_meta_df['visited_num'] = saved_meta_df['id'].apply(lambda x: item_count.loc[x] if x in item_count else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# user_history = df_train_0.groupby('user_id').agg(list)\n",
    "saved_meta_df.to_feather(os.path.join(chatbot_data_folder, 'games.ftr'))\n",
    "saved_meta_df = saved_meta_df.set_index('id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "max_title_len = 50\n",
    "id2title = {id: saved_meta_df.loc[id].title[: max_title_len] for id in saved_meta_df.index}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "N = 900\n",
    "test_data = df_test.sample(900)\n",
    "user_history = df_train_0[df_train_0['user_id'].isin(test_data['user_id'])].reset_index(drop=True).groupby('user_id').agg(list)\n",
    "max_len = 10\n",
    "test_data['history'] = test_data['user_id'].apply(lambda x: '; '.join([id2title[_] for _ in user_history.loc[x]['item_id'][-max_len:]]))\n",
    "test_data['target'] = test_data['item_id'].apply(lambda x: saved_meta_df.loc[x].title)\n",
    "test_data.reset_index(inplace=True, drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>item_id</th>\n",
       "      <th>history</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>201042</td>\n",
       "      <td>597</td>\n",
       "      <td>Loadout; Sins of a Solar Empire Rebellion; Rus...</td>\n",
       "      <td>HalfLife 2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>118030</td>\n",
       "      <td>1622</td>\n",
       "      <td>S.K.I.L.L.  Special Force 2 Shooter; Contagion...</td>\n",
       "      <td>TOXIKK</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13555</td>\n",
       "      <td>4101</td>\n",
       "      <td>Woodle Tree Adventures; Knightmare Tower; Shov...</td>\n",
       "      <td>Angry Video Game Nerd II ASSimilation</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5916</td>\n",
       "      <td>551</td>\n",
       "      <td>Streets of Rogue; b; Unforgiving Trials The Sp...</td>\n",
       "      <td>Soda Dungeon</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>164165</td>\n",
       "      <td>40</td>\n",
       "      <td>Imperial Glory; Mount  Blade Warband; Age of E...</td>\n",
       "      <td>Dino DDay</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>895</th>\n",
       "      <td>84755</td>\n",
       "      <td>2026</td>\n",
       "      <td>South Park The Stick of Truth; Antichamber; Ca...</td>\n",
       "      <td>Reigns</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>896</th>\n",
       "      <td>206543</td>\n",
       "      <td>70</td>\n",
       "      <td>The Repopulation; RimWorld; Cloudbuilt; Osiris...</td>\n",
       "      <td>No Mans Sky</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>897</th>\n",
       "      <td>143863</td>\n",
       "      <td>4106</td>\n",
       "      <td>Alan Wake; Shadow Warrior Classic Redux; Ace o...</td>\n",
       "      <td>Feel The Snow</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>898</th>\n",
       "      <td>68368</td>\n",
       "      <td>39</td>\n",
       "      <td>F.E.A.R.; The Witcher 2 Assassins of Kings Enh...</td>\n",
       "      <td>Deus Ex Human Revolution  Directors Cut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>899</th>\n",
       "      <td>145615</td>\n",
       "      <td>2899</td>\n",
       "      <td>Super Hexagon; Waste Walkers; Moonstone Tavern...</td>\n",
       "      <td>Owlboy</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>900 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id  item_id                                            history  \\\n",
       "0     201042      597  Loadout; Sins of a Solar Empire Rebellion; Rus...   \n",
       "1     118030     1622  S.K.I.L.L.  Special Force 2 Shooter; Contagion...   \n",
       "2      13555     4101  Woodle Tree Adventures; Knightmare Tower; Shov...   \n",
       "3       5916      551  Streets of Rogue; b; Unforgiving Trials The Sp...   \n",
       "4     164165       40  Imperial Glory; Mount  Blade Warband; Age of E...   \n",
       "..       ...      ...                                                ...   \n",
       "895    84755     2026  South Park The Stick of Truth; Antichamber; Ca...   \n",
       "896   206543       70  The Repopulation; RimWorld; Cloudbuilt; Osiris...   \n",
       "897   143863     4106  Alan Wake; Shadow Warrior Classic Redux; Ace o...   \n",
       "898    68368       39  F.E.A.R.; The Witcher 2 Assassins of Kings Enh...   \n",
       "899   145615     2899  Super Hexagon; Waste Walkers; Moonstone Tavern...   \n",
       "\n",
       "                                      target  \n",
       "0                                 HalfLife 2  \n",
       "1                                     TOXIKK  \n",
       "2      Angry Video Game Nerd II ASSimilation  \n",
       "3                               Soda Dungeon  \n",
       "4                                  Dino DDay  \n",
       "..                                       ...  \n",
       "895                                   Reigns  \n",
       "896                              No Mans Sky  \n",
       "897                            Feel The Snow  \n",
       "898  Deus Ex Human Revolution  Directors Cut  \n",
       "899                                   Owlboy  \n",
       "\n",
       "[900 rows x 4 columns]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import *\n",
    "import json, pickle\n",
    "\n",
    "def write_jsonl(obj: List[Dict], fpath: str) -> None:\n",
    "    try:\n",
    "        with open(fpath, 'w') as outfile:\n",
    "            for entry in obj:\n",
    "                json.dump(entry, outfile)\n",
    "                outfile.write('\\n')\n",
    "        print(\"Sucessfully saved into {}.\".format(fpath))\n",
    "    except Exception as e:\n",
    "        print(f\"Error {e} raised. The temp file would be saved in {fpath}.pkl\")\n",
    "        with open(f\"{fpath}.pkl\", 'wb') as tempfile:\n",
    "            pickle.dump(obj, tempfile)\n",
    "    return\n",
    "\n",
    "test_data_jsonl = test_data[['history', 'target']].to_dict(\"records\")\n",
    "write_jsonl(test_data_jsonl, os.path.join(chatbot_data_folder, f\"simulator_test_data_{N}.jsonl\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Bad pipe message: %s [b'S\\x97\\xa0w\\x95\\xa3\\xd0\\xad:\\x1b\\x02\\xea5\\xa0\\x81X\\xbd\\xd3 \\x0b3\\xc7\\xb71\\xe1\\x1d\\xe3*\\x1c\\xc2@\\x9a0\\xbb\\x82\\x0f\\x13\\xec\\x01\\x84\\xa2\\xcfM\\xda\\x930\\x1ax\\xa6']\n",
      "Bad pipe message: %s [b'\\x00\\x08\\x13\\x02\\x13\\x03\\x13\\x01\\x00\\xff\\x01\\x00\\x00\\x8f\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00', b'\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x00\\x1e\\x00\\x1c\\x04\\x03\\x05']\n",
      "Bad pipe message: %s [b'\\x03\\x08']\n",
      "Bad pipe message: %s [b\"3\\xd6{\\xc1\\xfb\\x80\\r\\x1cg\\xb0\\xb0\\xcf\\x94\\xbc]\\xafd\\xb2 \\xe7n$\\x80x'\\xe3!\", b'g\\x1ev\\xd2S\\xb3\\xab\\xc0\\xf5lfr\\xe7@\\x81Sj\\xe09F\\x8b\\x1f\\x00\\x08\\x13\\x02\\x13\\x03\\x13\\x01\\x00\\xff\\x01\\x00\\x00\\x8f\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16']\n",
      "Bad pipe message: %s [b'\\x08\\x08\\t\\x08\\n\\x08']\n",
      "Bad pipe message: %s [b'g[\\x8b]\\xa4}\\xfb^}P\\xaby\\xaf)b\\xbd\\xf1\\xe9\\x00\\x00\\xa6\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00', b\"s\\xc0w\\x00\\xc4\\x00\\xc3\\xc0#\\xc0'\\x00g\\x00@\\xc0r\\xc0v\\x00\\xbe\\x00\\xbd\\xc0\\n\\xc0\\x14\\x009\\x008\\x00\\x88\\x00\\x87\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9a\\x00\\x99\\x00E\\x00D\\xc0\\x07\\xc0\\x11\\xc0\\x08\\xc0\\x12\\x00\\x16\\x00\\x13\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00\\xc0\\x00<\\x00\\xba\\x005\\x00\\x84\\x00/\\x00\\x96\\x00A\\x00\\x05\\x00\\n\\x00\\xff\\x01\\x00\"]\n",
      "Bad pipe message: %s [b\"\\xc1Z\\xeb\\xa3\\xdf\\xb2+\\xa6s\\xb8\\xae\\xc64\\xdd\\x8d/\\xdfg\\x00\\x00|\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0#\\xc0'\\x00g\\x00@\\xc0\\n\\xc0\\x14\\x009\\x008\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00<\\x005\\x00/\\x00\\x9a\\x00\\x99\\xc0\\x07\\xc0\\x11\\x00\\x96\\x00\\x05\\x00\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.\", b'0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x000\\x00.\\x04\\x03']\n",
      "Bad pipe message: %s [b'\\x04\\x08\\x05\\x08\\x06\\x04\\x01\\x05\\x01\\x06']\n",
      "Bad pipe message: %s [b'\\x06\\x03\\x08\\x07', b'\\x08\\t\\x08\\n\\x08\\x0b\\x08']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b'\\x05\\x08\\x06']\n",
      "Bad pipe message: %s [b'\\x03\\x02\\x03\\x04\\x00-\\x00\\x02\\x01\\x01\\x003\\x00&\\x00$\\x00\\x1d\\x00 o]\\xf9\\xa2\\x9a\\xc3\\xc8d\\xcd\\x97\\x82\\x8a$TP\\x96\\x85\\x92\\xe8\\x01RK']\n",
      "Bad pipe message: %s [b'\\x05\\x01\\x06', b'', b'\\x03\\x03']\n",
      "Bad pipe message: %s [b'\\xcag\\x89\\x91D\\xe5S\\xc2\\xb3G]\\x97/\\xcb\\xc0\\xa0\\xb7\\x05\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\\x0f\\x00\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00\\x06\\x00\\x17\\x00\\x03\\xc0\\x10\\xc0\\x06\\xc0\\x15\\xc0\\x0b\\xc0\\x01\\x00\\x02\\x00\\x01\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t']\n",
      "Bad pipe message: %s [b'7.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x1c\\x00\\x1a\\x00\\x17\\x00\\x19\\x00\\x1c\\x00\\x1b\\x00\\x18\\x00\\x1a\\x00\\x16\\x00\\x0e\\x00\\r\\x00\\x0b\\x00\\x0c\\x00\\t\\x00\\n\\x00']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b'', b'\\x02']\n",
      "Bad pipe message: %s [b'\\x05\\x02\\x06']\n",
      "Bad pipe message: %s [b'R\\x82x\\xe0\\nU\\xac\\xa3\\xb6\\x9a\\xa5\\xb8y\\xcd\\xdaN-\\x04\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n']\n",
      "Bad pipe message: %s [b'B\\xbd\\x8fc\\xb0\\x83{\\xe0Q\\x02\\xf8\\x87\\x84\\xc3\\xb3\\xf2\\xde#\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0']\n",
      "Bad pipe message: %s [b'\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0']\n",
      "Bad pipe message: %s [b'\\x16\\x00\\x13\\x00\\x10\\x00\\r']\n",
      "Bad pipe message: %s [b\"\\x08\\xd8#W2:\\x9d\\x87p\\xb7\\x8c\\x9f\\xc9\\xeds'\\x90\\x85\\x00\\x00\\xa2\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\", b'\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00']\n",
      "Bad pipe message: %s [b'\\x17\\x00\\x03\\xc0\\x10']\n",
      "Bad pipe message: %s [b\"g\\x9e\\x9c\\x0b\\xc2\\xabH\\xba{9n\\x90\\x9cP\\xc5\\x11\\xedw\\x00\\x00\\x86\\xc00\\xc0,\\xc0(\\xc0$\\xc0\\x14\\xc0\\n\\x00\\xa5\\x00\\xa3\\x00\\xa1\\x00\\x9f\\x00k\\x00j\\x00i\\x00h\\x009\\x008\\x007\\x006\\xc02\\xc0.\\xc0*\\xc0&\\xc0\\x0f\\xc0\\x05\\x00\\x9d\\x00=\\x005\\xc0/\\xc0+\\xc0'\\xc0#\\xc0\\x13\\xc0\\t\\x00\\xa4\\x00\\xa2\\x00\\xa0\\x00\\x9e\\x00g\\x00@\\x00?\\x00>\\x003\\x002\\x001\\x000\\xc01\\xc0-\\xc0)\\xc0%\\xc0\\x0e\\xc0\\x04\\x00\\x9c\\x00<\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\", b'\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00']\n",
      "Bad pipe message: %s [b'\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x1c\\x00\\x1a\\x00\\x17\\x00\\x19\\x00\\x1c\\x00\\x1b\\x00\\x18\\x00\\x1a\\x00\\x16\\x00\\x0e\\x00\\r\\x00\\x0b\\x00\\x0c\\x00\\t\\x00\\n\\x00#\\x00\\x00\\x00\\r\\x00 \\x00\\x1e\\x06\\x01\\x06\\x02\\x06\\x03\\x05\\x01\\x05\\x02\\x05\\x03\\x04\\x01\\x04\\x02\\x04\\x03\\x03\\x01\\x03\\x02\\x03\\x03\\x02\\x01\\x02\\x02\\x02\\x03\\x00\\x0f\\x00\\x01\\x01']\n",
      "Bad pipe message: %s [b'H>\\x11\\xc8\\x8f\\x1d\\xc1\\xbb\\x9f\\x99\\xc3\\x9b\\xd6Y\\xcc\\xc0L\\xc8 \\xf6C\\xbf\\xe9oM\\x08\\xd7\\xe8\\x9a:', b'^\\x9c\\xe1\\x0b\\xddf\\xe7\\x10\\x1e6\\xb9!\\x1e\\x80&\\x17\\xa5@\\xce\\x00\\x08\\x13\\x02\\x13\\x03\\x13\\x01\\x00\\xff\\x01\\x00\\x00\\x8f\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x00\\x1e\\x00\\x1c\\x04\\x03\\x05\\x03\\x06\\x03\\x08\\x07\\x08\\x08', b'\\x08\\n\\x08\\x0b\\x08\\x04\\x08', b'\\x06\\x04\\x01\\x05']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b'\\x1b@>K\\xe3\\xc1\\\\\\xee\\xe8\\xb2d\\xb76\\xae\\xb7P*\\xd7 \\xf1\\x19\\x1e\\xa8\\xa9`T|\\xb8\\xc9B\\xfe\\xe7\\x97\"\\xeeR\\x8e\\xe9\\xa1\\xa6q\\x0bi\\x90\\x92']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b'\\x03\\x02\\x03\\x04\\x00-\\x00\\x02\\x01\\x01\\x003\\x00&\\x00$\\x00\\x1d\\x00 \\xeehV\\xff\\xbe\\xbcE\\xab\\x02\\xb9mbALq\\xc4\\x1f\"\\xcc\\x94\\x1c\\xf7']\n",
      "Bad pipe message: %s [b\"\\xee\\xa11\\x83\\xd7\\xe4\\x1d?\\x922\\xa3\\xc8\\x84\\xc7\\xd5\\xfe\\xab>\\x00\\x00|\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0#\\xc0'\\x00g\\x00@\\xc0\\n\\xc0\\x14\\x009\\x008\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00<\\x005\\x00/\\x00\\x9a\\x00\\x99\\xc0\\x07\\xc0\\x11\\x00\\x96\\x00\\x05\\x00\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\\x00\\x0c\\x00\\n\\x00\\x1d\\x00\\x17\\x00\\x1e\\x00\\x19\\x00\\x18\\x00#\\x00\\x00\\x00\\x16\\x00\\x00\\x00\\x17\\x00\\x00\\x00\\r\\x000\\x00.\\x04\\x03\\x05\\x03\\x06\\x03\\x08\\x07\\x08\\x08\\x08\\t\\x08\\n\\x08\\x0b\\x08\\x04\\x08\\x05\", b'\\x04\\x01\\x05\\x01\\x06\\x01\\x03']\n",
      "Bad pipe message: %s [b'\\x03\\x03']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b\"\\xcd\\xd6\\xd0\\xdc\\xcb(\\x17\\xe3 \\xc7O\\x08\\x17Q<]\\xed\\x89\\x00\\x00\\xa6\\xc0,\\xc00\\x00\\xa3\\x00\\x9f\\xcc\\xa9\\xcc\\xa8\\xcc\\xaa\\xc0\\xaf\\xc0\\xad\\xc0\\xa3\\xc0\\x9f\\xc0]\\xc0a\\xc0W\\xc0S\\xc0+\\xc0/\\x00\\xa2\\x00\\x9e\\xc0\\xae\\xc0\\xac\\xc0\\xa2\\xc0\\x9e\\xc0\\\\\\xc0`\\xc0V\\xc0R\\xc0$\\xc0(\\x00k\\x00j\\xc0s\\xc0w\\x00\\xc4\\x00\\xc3\\xc0#\\xc0'\\x00g\\x00@\\xc0r\\xc0v\\x00\\xbe\\x00\\xbd\\xc0\\n\\xc0\\x14\\x009\\x008\\x00\\x88\\x00\\x87\\xc0\\t\\xc0\\x13\\x003\\x002\\x00\\x9a\\x00\\x99\\x00E\\x00D\\xc0\\x07\\xc0\\x11\\xc0\\x08\\xc0\\x12\\x00\\x16\\x00\\x13\\x00\\x9d\\xc0\\xa1\\xc0\\x9d\\xc0Q\\x00\\x9c\\xc0\\xa0\\xc0\\x9c\\xc0P\\x00=\\x00\\xc0\\x00<\\x00\\xba\\x005\\x00\\x84\\x00/\\x00\\x96\\x00A\\x00\\x05\\x00\\n\\x00\\xff\\x01\\x00\\x00j\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02\\x00\\n\"]\n",
      "Bad pipe message: %s [b'', b'\\x02']\n",
      "Bad pipe message: %s [b'\\x05\\x02\\x06']\n",
      "Bad pipe message: %s [b'\\x8a\\x9dG\\x07<?\\xc3\\xde%8(\\x86\\x1a\\x05-\\xf1\\xd7A\\x00\\x00\\xa2', b'\\xc0\\n\\x009\\x008\\x007\\x006\\x00\\x88\\x00\\x87\\x00\\x86\\x00\\x85\\xc0\\x19\\x00:\\x00\\x89\\xc0\\x0f\\xc0\\x05\\x005\\x00\\x84\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00E\\x00D\\x00C\\x00B\\xc0\\x18\\x004\\x00\\x9b\\x00F\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x96\\x00A\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x16\\x00\\x18\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\xc0\\x12\\xc0\\x08\\x00\\x16\\x00\\x13\\x00\\x10\\x00\\r\\xc0\\x17\\x00\\x1b\\xc0\\r\\xc0\\x03\\x00\\n\\x00\\x15\\x00\\x12\\x00\\x0f\\x00\\x0c\\x00\\x1a\\x00\\t\\x00\\x14\\x00\\x11\\x00\\x19\\x00\\x08\\x00\\x06\\x00\\x17\\x00\\x03\\xc0\\x10\\xc0\\x06\\xc0\\x15\\xc0\\x0b\\xc0\\x01\\x00\\x02\\x00\\x01\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00\\x01\\x02']\n",
      "Bad pipe message: %s [b')\\x02\\x00\\xd0\\x1by\\x82b\\xa3\\xeb$\\xff9X\\x06\\x89\\xa8\\r\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c\\xc0\\x02\\x00\\x05\\x00\\x04\\x00\\xff\\x02\\x01\\x00\\x00C\\x00\\x00\\x00\\x0e\\x00\\x0c\\x00\\x00\\t127.0.0.1\\x00\\x0b\\x00\\x04\\x03\\x00']\n",
      "Bad pipe message: %s [b'']\n",
      "Bad pipe message: %s [b'>\\xfc\\xb17\\x97\\x956\\n\\xf5\\x90T\\xd3\\xc3o\\x99\\x0c\\x9c\\xfe\\x00\\x00>\\xc0\\x14\\xc0\\n\\x009\\x008\\x007\\x006\\xc0\\x0f\\xc0\\x05\\x005\\xc0\\x13\\xc0\\t\\x003\\x002\\x001\\x000\\xc0\\x0e\\xc0\\x04\\x00/\\x00\\x9a\\x00\\x99\\x00\\x98\\x00\\x97\\x00\\x96\\x00\\x07\\xc0\\x11\\xc0\\x07\\xc0\\x0c']\n"
     ]
    }
   ],
   "source": [
    "test_data['history'].apply(lambda x: len(x.split(\"; \"))).max()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "llm4crs",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
